{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "15e1e6a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession,Row\n",
    "from pyspark.sql.functions import col,locate,regexp_extract\n",
    "from pyspark.sql.types import StructType,StructField, StringType,ArrayType,IntegerType\n",
    "from typing import List, Tuple\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c05a4860",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1281035\n"
     ]
    }
   ],
   "source": [
    "sentences = \"\"\n",
    "with open(\"sentences.txt\", \"r\", encoding = 'utf-8') as f:\n",
    "    sentences = f.read().splitlines() \n",
    "print(len(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2f2a7c9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/apache-spark/3.2.1/libexec/conf/spark-env.sh: line 1: spark.executor.memory=6g: command not found\n",
      "2022-04-22 00:06:52,724 WARN util.Utils: Your hostname, huzujundeMacBook-Pro.local resolves to a loopback address: 127.0.0.1; using 192.168.3.6 instead (on interface en0)\n",
      "2022-04-22 00:06:52,725 WARN util.Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/usr/local/Cellar/apache-spark/3.2.1/libexec/jars/spark-unsafe_2.12-3.2.1.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "2022-04-22 00:06:53,385 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "2022-04-22 00:06:54,366 WARN util.Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n",
      "2022-04-22 00:06:54,366 WARN util.Utils: Service 'SparkUI' could not bind on port 4041. Attempting port 4042.\n"
     ]
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName(\"createTable\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "58fa18fb",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/n4/g1xtbtsn5gz6k5yv4ss93hfc0000gn/T/ipykernel_89067/2983395661.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mpypinyin\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mlazy_pinyin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mStyle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/anaconda3/envs/python39/lib/python3.8/site-packages/pypinyin/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpypinyin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPY2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m from pypinyin.constants import (  # noqa\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mStyle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mSTYLE_NORMAL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNORMAL\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/envs/python39/lib/python3.8/site-packages/pypinyin/constants.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mPHRASES_DICT\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0;32mfrom\u001b[0m \u001b[0mpypinyin\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mphrases_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mPHRASES_DICT\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mphrases_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mphrases_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/envs/python39/lib/python3.8/site-packages/pypinyin/phrases_dict.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m   4225\u001b[0m     \u001b[0;34m'优待券'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'yōu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'dài'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'quàn'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4226\u001b[0m     \u001b[0;34m'优惠券'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'yōu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'huì'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'quàn'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4227\u001b[0;31m     \u001b[0;34m'优惠待遇'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'yōu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'huì'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'dài'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'yù'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4228\u001b[0m     \u001b[0;34m'优游不断'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'yōu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'yóu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'bù'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'duàn'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4229\u001b[0m     \u001b[0;34m'优游卒岁'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'yōu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'yóu'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'zú'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'suì'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from pypinyin import lazy_pinyin, Style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "89faf82a",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['a', 'en', 'ong', 'II', 'ei', 'e', 'ian', 'a', 'in', 'u', 'i', 'in'],\n",
       " ['a1',\n",
       "  'en',\n",
       "  'ong3',\n",
       "  'II4',\n",
       "  'ei4',\n",
       "  'e',\n",
       "  'ian4',\n",
       "  'a4',\n",
       "  'in2',\n",
       "  'u4',\n",
       "  'i4',\n",
       "  'in2'])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def word_parser(sentence: str) -> Tuple[List[str], List[str]]:\n",
    "    pinyins = lazy_pinyin(sentence)\n",
    "    rhys = pinyin_parser(pinyins)\n",
    "    rhys_tone = add_tone(sentence, rhys)\n",
    "    return rhys, rhys_tone\n",
    "\n",
    "def add_tone(sentence, ryhs):\n",
    "    pinyins_tone = lazy_pinyin(sentence, style=Style.FINALS_TONE3)\n",
    "    result = []\n",
    "    for i, ryh in enumerate(ryhs):\n",
    "        if pinyins_tone[i][-1] == '1' or pinyins_tone[i][-1] == '2' or pinyins_tone[i][-1] == '3' or pinyins_tone[i][-1] == '4':\n",
    "            result.append(ryh+pinyins_tone[i][-1])\n",
    "        else:\n",
    "            result.append(ryh)\n",
    "    return result\n",
    "\n",
    "\n",
    "def pinyin_parser(pinyins: List[str]) -> List[List[str]]:\n",
    "    parsed_pinyins = []\n",
    "    for pinyin in pinyins:\n",
    "        pinyin = pinyin_correction(pinyin)\n",
    "        consonant, vowel = split_cv(pinyin)\n",
    "        parsed_pinyins.append(weak_rhyme(vowel_parser(vowel)))\n",
    "    return parsed_pinyins\n",
    "\n",
    "\n",
    "def pinyin_correction(pinyin: str) -> str:\n",
    "\n",
    "    # z/c/s + i -> z/c/s + I\n",
    "    if re.match(r'[zcs]i$', pinyin):\n",
    "        return pinyin.replace('i', 'I')\n",
    "\n",
    "    # zh/ch/sh/r + i -> zh/ch/sh/r + II\n",
    "    elif re.match(r'(?:zh|ch|sh|r)i$', pinyin):\n",
    "        return pinyin.replace('i', 'II')\n",
    "\n",
    "    # j/q/x/y + u/ue/un/uan -> j/q/x + v/ve/vn/van | v/ve/vn/van\n",
    "    # yu == yi\n",
    "    elif re.match(r'[jqxy]u', pinyin):\n",
    "        if pinyin.endswith('u'):\n",
    "            return re.sub(r'y*(.+)', r'\\1', pinyin.replace('u', 'i'))\n",
    "        return re.sub(r'y*(.+)', r'\\1', pinyin.replace('u', 'v'))\n",
    "\n",
    "    # y + a/e/ao/ou/an/ in/iang/ing/iong -> ia/ie/iao/iou/ian/ in/iang/ing/iong\n",
    "    elif pinyin.startswith(\"y\"):\n",
    "        return re.sub(r'yi*(.*)', r'i\\1', pinyin)\n",
    "\n",
    "    # w + u/a/o/ai/ei/an/en/ang/eng        -> u/ua/uo/uai/uei/uan/uen/uang/ueng\n",
    "    elif pinyin.startswith(\"w\"):\n",
    "        return re.sub(r'wu*(.*)', r'u\\1', pinyin)\n",
    "\n",
    "    # qiu -> qiou\n",
    "    elif pinyin.endswith('iu'):\n",
    "        return pinyin.replace('iu', 'iou')\n",
    "\n",
    "    # cui -> cuei\n",
    "    elif pinyin.endswith('ui'):\n",
    "        return pinyin.replace('ui', 'uei')\n",
    "\n",
    "    # lun -> luen\n",
    "    elif pinyin.endswith('un'):\n",
    "        return pinyin.replace('un', 'uen')\n",
    "\n",
    "\n",
    "    return pinyin\n",
    "\n",
    "\n",
    "def split_cv(pinyin):\n",
    "    return re.findall(r'(ch|zh|sh|[^aeiIouv])*(.+)', pinyin)[0]\n",
    "\n",
    "\n",
    "def vowel_parser(vowel: str) -> str:\n",
    "    if vowel == 'van':\n",
    "        return 'an'\n",
    "\n",
    "    elif len(vowel) > 1 and vowel[0] == 'u':\n",
    "        return vowel[1:]\n",
    "\n",
    "    # for 'i', except 'in' and 'ing' (one vowel), 'ie' and 'ian' (sound different from 'e' and 'an')\n",
    "    elif len(vowel) > 1 and vowel[0] == 'i' and vowel[1] != 'n' and vowel[1] != 'e' and vowel[1:] != 'an':\n",
    "        return vowel[1:]\n",
    "\n",
    "    return vowel\n",
    "\n",
    "def weak_rhyme(pinyin):\n",
    "    if pinyin.endswith('eng'):\n",
    "        return pinyin.replace('eng', 'en')\n",
    "\n",
    "    elif pinyin.endswith('ang'):\n",
    "        return pinyin.replace('ang', 'an')\n",
    "\n",
    "    elif pinyin.endswith('ing'):\n",
    "        return pinyin.replace('ing', 'in')\n",
    "\n",
    "    elif pinyin.endswith('ui'):\n",
    "        return pinyin.replace('ui', 'ei')\n",
    "\n",
    "    elif pinyin.endswith('ie'):\n",
    "        return pinyin.replace('ui', 'ei')\n",
    "\n",
    "    elif pinyin.endswith('ue'):\n",
    "        return pinyin.replace('ue', 'ei')\n",
    "\n",
    "    return pinyin\n",
    "\n",
    "word_parser(sentences[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "815fde05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[\"'ou', 'in', 'e', 'o', 'e', 'ei', 'ao'\", \"'in', 'in', 'i', 'an', 'e', 'o', 'ong', 'i', 'en', 'ei', 'in', 'ei', 'i', 'i'\", \"'a', 'en', 'ong', 'II', 'ei', 'e', 'ian', 'a', 'in', 'u', 'i', 'in'\", \"'o', 'a', 'ai', 'ei', 'in', 'u', 'II', 'ai', 'in', 'i', 'e', 'ie', 'e'\", \"'en', 'ou', 'an', 'u', 'ou', 'in', 'ai', 'ao', 'i', 'u', 'ou', 'ai', 'ao', 'i'\", \"'ou', 'e', 'e', 'ian', 'u', 'ei', 'ian', 'a'\", \"'an', 'e', 'ou', 'en', 'ong', 'ian', 'ou'\", \"'an', 'ou', 'I', 'i', 'ao', 'i', 'i', 'o', 'an', 'an', 'u', 'ou'\", \"'en', 'e', 'en', 'ou', 'u', 'e', 'i', 'ao', 'o', 'i', 'ie', 'u', 'e', 'i'\", \"'o', 'ai', 'ou', 'II', 'in', 'i', 'o'\"] [\"'ou4', 'in3', 'e', 'o3', 'e', 'ei2', 'ao2'\", \"'in1', 'in2', 'i1', 'an1', 'e', 'o4', 'ong4', 'i1', 'en3', 'ei4', 'in2', 'ei4', 'i2', 'i3'\", \"'a1', 'en', 'ong3', 'II4', 'ei4', 'e', 'ian4', 'a4', 'in2', 'u4', 'i4', 'in2'\", \"'o3', 'a3', 'ai1', 'ei1', 'in4', 'u2', 'II4', 'ai2', 'in1', 'i3', 'e4', 'ie1', 'e'\", \"'en2', 'ou3', 'an1', 'u1', 'ou4', 'in2', 'ai2', 'ao1', 'i1', 'u2', 'ou3', 'ai4', 'ao1', 'i2'\", \"'ou1', 'e', 'e4', 'ian3', 'u1', 'ei4', 'ian4', 'a4'\", \"'an4', 'e4', 'ou3', 'en3', 'ong2', 'ian2', 'ou4'\", \"'an3', 'ou3', 'I1', 'i1', 'ao4', 'i1', 'i1', 'o4', 'an1', 'an1', 'u3', 'ou4'\", \"'en4', 'e2', 'en2', 'ou1', 'u4', 'e3', 'i3', 'ao1', 'o4', 'i3', 'ie3', 'u4', 'e3', 'i3'\", \"'o3', 'ai2', 'ou3', 'II4', 'in2', 'i4', 'o4'\"]\n"
     ]
    }
   ],
   "source": [
    "from pypinyin import lazy_pinyin, Style\n",
    "pinyin_with_tone = []\n",
    "pinyin_without_tone = []\n",
    "n = len(sentences)\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    if (i%100000 == 0):\n",
    "        print(i)\n",
    "    with_tone, without_tone = word_parser(sentences[i])\n",
    "    pinyin_without_tone.append(str(with_tone)[1:-1])\n",
    "    pinyin_with_tone.append(str(without_tone)[1:-1])\n",
    "print(pinyin_without_tone, pinyin_with_tone)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "820494d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_find_all(s):\n",
    "    result = []\n",
    "    for i in range(len(s)):\n",
    "        if s[i] == ',':\n",
    "            result.append(i)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "964d9756",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_find_all(pinyin_without_tone[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d1e9ebb",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for i in range(n):\n",
    "    data.append(Row(sentences[i],pinyin_without_tone[i],my_find_all(pinyin_without_tone[i]),\n",
    "                    pinyin_with_tone[i],my_find_all(pinyin_with_tone[i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80a4e0ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "SentenceSchema = StructType([       \n",
    "    StructField('sentences', StringType(), True),\n",
    "    #StructField('pinyin_without_tone', ArrayType(StringType()), True),\n",
    "    #StructField('pinyin_with_tone',  ArrayType(StringType()), True)\n",
    "    StructField('pinyin_without_tone', StringType(), True),\n",
    "    StructField('pinyin_without_tone_split',  ArrayType(IntegerType()), True),\n",
    "    StructField('pinyin_with_tone',  StringType(), True),\n",
    "    StructField('pinyin_with_tone_split',  ArrayType(IntegerType()), True)\n",
    "])\n",
    "\n",
    "df = spark.createDataFrame(data=data, schema = SentenceSchema)\n",
    "df.printSchema()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe35296",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write.option(\"header\",True).format(\"orc\").mode('overwrite')\\\n",
    "    .save(\"rhyme/sentences.orc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fe48c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pycantonese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3a7aec3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "155617\n"
     ]
    }
   ],
   "source": [
    "sentences = \"\"\n",
    "with open(\"cantonese_sentences.txt\", \"r\", encoding = 'utf-8') as f:\n",
    "    sentences = f.read().splitlines() \n",
    "print(len(sentences))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d2ed215e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'过去十八岁没戴表不过有时间'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "de509859",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████| 155617/155617 [00:33<00:00, 4606.16it/s]\n"
     ]
    }
   ],
   "source": [
    "import pycantonese\n",
    "import zhconv\n",
    "from tqdm import tqdm, trange\n",
    "lpys = []\n",
    "lpys_notone = []\n",
    "s = []\n",
    "for i in trange(len(sentences)):\n",
    "    pys = pycantonese.characters_to_jyutping(zhconv.convert(sentences[i], 'zh-tw'))\n",
    "    pys_new = []\n",
    "    pys_new_notone = []\n",
    "    for py in pys:\n",
    "        for jp in pycantonese.parse_jyutping(py[1]):     \n",
    "            pys_new.append(jp.final + jp.tone)\n",
    "            pys_new_notone.append(jp.final)\n",
    "    #pinyin.append(str(pys_new))\n",
    "    if len(pys_new) == len(pys_new_notone) and len(pys_new_notone) == len(sentences[i]):\n",
    "        s.append(sentences[i])\n",
    "        lpys.append(str(pys_new)[1:-1])\n",
    "        lpys_notone.append(str(pys_new_notone)[1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "72fffad4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "155467\n"
     ]
    }
   ],
   "source": [
    "print(len(lpys))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5678ba75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 10, 16, 23, 30, 36, 43, 49, 55, 60, 66, 71]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_find_all(s):\n",
    "    result = []\n",
    "    for i in range(len(s)):\n",
    "        if s[i] == ',':\n",
    "            result.append(i)\n",
    "    return result\n",
    "my_find_all(lpys_notone[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8fa46b95",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████| 155467/155467 [00:02<00:00, 68143.91it/s]\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "for i in trange(len(s)):\n",
    "    data.append(Row(s[i],lpys_notone[i],my_find_all(lpys_notone[i]),\n",
    "                    lpys[i],my_find_all(lpys[i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3e2d1f6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Row('吞风吻雨葬落日未曾徬徨', \"'an', 'ung', 'an', 'yu', 'ong', 'ok', 'at', 'ei', 'ang', 'ong', 'ong'\", [4, 11, 17, 23, 30, 36, 42, 48, 55, 62], \"'an1', 'ung1', 'an5', 'yu5', 'ong3', 'ok6', 'at6', 'ei6', 'ang4', 'ong4', 'ong4'\", [5, 13, 20, 27, 35, 42, 49, 56, 64, 72])>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e891097d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- sentences: string (nullable = true)\n",
      " |-- pinyin_without_tone: string (nullable = true)\n",
      " |-- pinyin_without_tone_split: array (nullable = true)\n",
      " |    |-- element: integer (containsNull = true)\n",
      " |-- pinyin_with_tone: string (nullable = true)\n",
      " |-- pinyin_with_tone_split: array (nullable = true)\n",
      " |    |-- element: integer (containsNull = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "SentenceSchema = StructType([       \n",
    "    StructField('sentences', StringType(), True),\n",
    "    #StructField('pinyin_without_tone', ArrayType(StringType()), True),\n",
    "    #StructField('pinyin_with_tone',  ArrayType(StringType()), True)\n",
    "    StructField('pinyin_without_tone', StringType(), True),\n",
    "    StructField('pinyin_without_tone_split',  ArrayType(IntegerType()), True),\n",
    "    StructField('pinyin_with_tone',  StringType(), True),\n",
    "    StructField('pinyin_with_tone_split',  ArrayType(IntegerType()), True)\n",
    "])\n",
    "\n",
    "df = spark.createDataFrame(data=data, schema = SentenceSchema)\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ab50259",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2e457847",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-22 00:24:57,721 WARN scheduler.TaskSetManager: Stage 1 contains a task of very large size (3971 KiB). The maximum recommended task size is 1000 KiB.\n",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df.write.option(\"header\",True).format(\"orc\").mode('overwrite')\\\n",
    "    .save(\"rhyme/cantonese.orc\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
